#include "../../../include/offline/CreateLib.hpp"
#include "../../../include/simhash/Simhasher.hpp"
#include "../../../include/cppjieba/Jieba.hpp"
#include "../../../include/tinyxml2/tinyxml2.h"


using namespace std;
using namespace tinyxml2;
using namespace simhash;


const char* const DICT_PATH = "../include/cppjieba/dict/jieba.dict.utf8";
const char* const HMM_PATH = "../include/cppjieba/dict/hmm_model.utf8";
const char* const USER_DICT_PATH = "../include/cppjieba/dict/user.dict.utf8";
const char* const IDF_PATH = "../include/cppjieba/dict/idf.utf8";
const char* const STOP_WORD_PATH = "../include/cppjieba/dict/stop_words.utf8";


PageLib::PageLib() {}

PageLib::PageLib(DirScanner& dirScanner) : _dirScanner(dirScanner) {}

void PageLib::readOriXml(const string &filename) {
    XMLDocument doc;
    doc.LoadFile(filename.c_str());
    if(doc.ErrorID()) {
        cerr << "loadFile fail" << endl;
        return;
    }

    XMLElement *itemNode = doc.FirstChildElement("rss")->FirstChildElement("channel")->FirstChildElement("item");
    int idx = 0;
    while (itemNode) {
        Page webPage;

        if (itemNode->FirstChildElement("title") == nullptr) {
            itemNode = itemNode->NextSiblingElement("item");
            break;
        } else {
            string title = itemNode->FirstChildElement("title")->GetText();

            regex reg("<[^>]+>");
            title = regex_replace(title, reg, "");
            
            regex reg2("[\\s|\n\t\r^&$]");
            title = regex_replace(title, reg2, "");

            if (title.empty()) {
                itemNode = itemNode->NextSiblingElement("item");
                break;
            }

            webPage._title = title;
        }

        if (itemNode->FirstChildElement("link") == nullptr) {
            itemNode = itemNode->NextSiblingElement("item");
            break;
        } else {
            string url=  itemNode->FirstChildElement("link")->GetText();

            regex reg("<[^>]+>");
            url = regex_replace(url, reg, "");
            
            regex reg2("[\\s|\n\t\r^&$]");
            url = regex_replace(url, reg2, "");

            if (url.empty()) {
                itemNode = itemNode->NextSiblingElement("item");
                break;
            }

            webPage._url = url;
        }

        if (itemNode->FirstChildElement("description") == nullptr) {
            itemNode = itemNode->NextSiblingElement("item");
            break;
        } else {
            string content = itemNode->FirstChildElement("description")->GetText();
            
            regex reg("<[^>]+>");
            content = regex_replace(content, reg, "");
            
            regex reg2("[\\s|\n\t\r^&$]");
            content = regex_replace(content, reg2, "");

            if (content.empty()) {
                itemNode = itemNode->NextSiblingElement("item");
                break;
            }
            webPage._content = content;
        }
        
        _page.push_back(webPage);
        ++idx; 

        itemNode = itemNode->NextSiblingElement("item");
    }
}


void PageLib::generateOriLib() {
    // ofstream ofsPage("../../../data/dat/ripePage.dat");
    ofstream ofsPage("../data/dat/ripePage.dat");
    if (!ofsPage) {
        cerr << "open ofsPage file failed" << endl;
        return;
    }

    ofstream ofsOffset("../data/dat/offset.dat");
    if (!ofsOffset) {
        cerr << "open ofsOffset file failed" << endl;
        return;
    }

    for (int i = 0; i != _page.size(); ++i) {
        int docid = i + 1;
        ofsOffset << docid << " ";
        size_t startPos = ofsPage.tellp();
        ofsOffset << startPos << " ";

        ofsPage << "<doc>\n"
                << "\t<docid>" << docid << "</docid>\n"
                << "\t<title>" << _page[i]._title << "</title>\n"
                << "\t<url>" << _page[i]._url << "</url>\n"
                << "\t<content>" << _page[i]._content << "</content>\n"
                << "</doc>";
        ofsPage << '\n';

        size_t endPos = ofsPage.tellp();
        size_t len = endPos - startPos;
        ofsOffset << len << '\n';
    }

    ofsOffset.close();
    ofsPage.close();
}


void PageLib::readNewXml(const string& filename) {
    XMLDocument doc;
    // doc.LoadFile("../../../data/dat/ripePage.dat");
    doc.LoadFile(filename.c_str());
    if (doc.ErrorID()) {
        cerr << "Load file failed." << endl;
        return;
    }

    XMLElement *itemNode = doc.FirstChildElement("doc");
    while (itemNode) {
        
        string docid = itemNode->FirstChildElement("docid")->GetText();
        string title = itemNode->FirstChildElement("title")->GetText();
        string url=  itemNode->FirstChildElement("url")->GetText();
        string content = itemNode->FirstChildElement("content")->GetText();

        regex reg("<[^>]+>|\\s");
        content = regex_replace(content, reg, "");

        Page newPage;
        newPage._title = title;
        newPage._url = url;
        newPage._content = content;

        vector<Page> detail;    
        detail.push_back(newPage);

        // cast docid from string to int
        const char* p = docid.c_str();
        int id = 0;
        id = atoi(p);

        _ripePage.insert(pair<int, vector<Page>>(id, detail));
        
        itemNode = itemNode->NextSiblingElement("doc");
    }
}


void PageLib::calculateSimHash() {
    Simhasher simhasher("../include/simhash/dict/jieba.dict.utf8",
                        "../include/simhash/dict/hmm_model.utf8",
                        "../include/simhash/dict/idf.utf8",
                        "../include/simhash/dict/stop_words.utf8");
    size_t topN = 5; // 抽取关键词个数

    int totalPages = _ripePage.size();
    map<int, vector<Page>>::iterator it = _ripePage.begin();
    for (; it != _ripePage.end(); it++) {
        uint64_t u64 = 0;

        auto itVec = it->second.begin();
        // cout << "docid = " << it->first << " url = " << itVec->_url << endl;
        
        simhasher.make(itVec->_content, topN, u64);
        _simHashMap.insert(pair<int, uint64_t>(it->first, u64));
    }

    /*test code*/
    // for (auto &t:_simHashMap) {
    //     cout << "docid = " << t.first << " simhash = " << t.second << endl;
    // }
}


void PageLib::similarityCheck() {
    int totalPages = _ripePage.size();

    // 1: 保留 0: 移除
    for (int i = 0; i < totalPages; i++) {
        // 初始化为全部保留
        _reserveIndex.push_back(1);      
    }

    // Iterators of ripaPage
    map<int, vector<Page>>::iterator it1 = _ripePage.begin(); // 前面一页
    map<int, vector<Page>>::iterator it2 = _ripePage.begin(); // 后面一页
    it2++;

    // Iterators of simHashMap
    map<int, uint64_t>::iterator h1 = _simHashMap.begin();
    map<int, uint64_t>::iterator h2 = _simHashMap.begin();
    h2++;

    for (; it1 != _ripePage.end(); it1++) {
        auto firstPage = it1->second.begin();

        for (; it2 != _ripePage.end(); it2++) {
            auto secondPage = it2->second.begin();
            if (secondPage->_url == firstPage->_url) { // if two pages have same url, delete the latter one
                _reserveIndex[it2->first] = 0;
            } else {
                if (Simhasher::isEqual(h1->first, h2->first)) { // if two pages have different url but same simhash value, delete the latter one
                    _reserveIndex[it2->first] = 0;
                }
            }
            h2++;
        }
        h1++;
    }

    /*test code*/
    // for (auto &m:_reserveIndex) {
    //     cout << m << endl;
    // }
}


void PageLib::generateNewLib() {
    map<int, vector<Page>>::iterator rpIt = _ripePage.begin();
    vector<int>::iterator riIt = _reserveIndex.begin();

    ofstream ofsPage("../data/dat/newRipePage.dat");
    if(!ofsPage) {
        cerr << "open newRipePage.dat failed" << endl;
        return;
    }

    ofstream ofsOffset("../data/dat/newOffset.dat");
    if (!ofsOffset) {
        cerr << "open newOffset.dat failed" << endl;
        return;
    }

    int docid = 1;

    for (auto &e:_reserveIndex) {
        auto detail = rpIt->second.begin();
        if (e == 1) {
            ofsOffset << docid << " ";
            size_t startPos = ofsPage.tellp();
            ofsOffset << startPos << " ";

            ofsPage << "<doc>\n"
                << "\t<docid>" << docid << "</docid>\n"
                << "\t<title>" << detail->_title << "</title>\n"
                << "\t<url>" << detail->_url << "</url>\n"
                << "\t<content>" << detail->_content << "</content>\n"
                << "</doc>";
            ofsPage << '\n';

            size_t endPos = ofsPage.tellp();
            size_t len = endPos - startPos;
            ofsOffset << len << '\n';

            docid++;
        }
        rpIt++;
    }
    ofsOffset.close();
    ofsPage.close();
}


int main() {

    // 生成未去重的dat文件
    cout << "=== Start getting file path ===" << endl << endl;
    string dirPath = "../data/corpus/module2";
    vector<string> files;
    DirScanner scanner;
    scanner.getFiles(dirPath);
    files = scanner.files();
    cout << "=== Got it! ===" << endl << endl;

    cout << "=== Start generating ripePage.dat and offset.dat files ===" << endl << endl;
    PageLib pageLib(scanner);
    for (auto &e:files) {
        cout << "read file " << e << endl;
        pageLib.readOriXml(e);
        cout << "generate file " << e << endl;
        pageLib.generateOriLib();
    }
    cout << "=== ripePage.dat and offset.dat were successfully generated ===" << endl << endl;

    // 生成去重后的dat文件
    cout << "=== Start generating newRipePage.dat and newOffset.dat files ===" << endl << endl;
    PageLib finalLib;
    finalLib.readNewXml("../data/dat/ripePage.dat");
    finalLib.calculateSimHash();
    finalLib.similarityCheck();
    finalLib.generateNewLib();
    cout << "=== newRipePage.dat and newOffset.dat were successfully generated ===" << endl << endl;

    return 0;
}